package com.avasthi.cli.twitter;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.file.Files;
import java.nio.file.StandardOpenOption;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Scanner;
import java.util.Set;

import de.l3s.boilerpipe.BoilerpipeProcessingException;
import de.l3s.boilerpipe.extractors.ArticleExtractor;
import net.minidev.json.JSONArray;
import net.minidev.json.JSONObject;
import net.minidev.json.JSONValue;
import net.minidev.json.parser.JSONParser;
import net.minidev.json.parser.ParseException;

public class TwitterJsonExtractHashtags {

	public static void main(String[] args) {
		// TODO Auto-generated method stub

		if (args.length == 0) {
			System.out.println("Usage: java TwitterJsonParser twitterlog");
			System.exit(0);
		}
		File inp = new File(args[0]);
		System.out.println("Reading file :" + inp.getAbsolutePath());
		try {
			FileReader fr = new FileReader(inp);
			BufferedReader reader = new BufferedReader(fr);
			JSONParser parser = new JSONParser(JSONParser.MODE_PERMISSIVE);
			JSONObject jv = null;
			PrintWriter hashtagsFile = new PrintWriter(
					"/home/vavasthi/tweets/hashtags.txt", "UTF-8");
			PrintWriter usersFile = new PrintWriter(
					"/home/vavasthi/tweets/users.txt", "UTF-8");
			PrintWriter usersOriginatorsFile = new PrintWriter(
					"/home/vavasthi/tweets/users-originators.txt", "UTF-8");
			PrintWriter usersMapFile = new PrintWriter(
					"/home/vavasthi/tweets/userMap.txt", "UTF-8");
			Map<String, Integer> hashtagCount = new HashMap<String, Integer>();
			Map<String, Integer> users = new HashMap<String, Integer>();
			Map<String, Integer> usersOriginators = new HashMap<String, Integer>();
			Map<UserPair, Integer> userMap = new HashMap<UserPair, Integer>();
			Object o;
			while ((o = parser.parse(reader)) != null) {
				try {
					
					jv = (JSONObject)o;
				}
				catch(ClassCastException cce) {
					break;
				}
				JSONObject currentUser = (JSONObject)(jv.get("user"));
				String sid = "";
				if (currentUser != null) {
					
					sid = (String) currentUser.get("id_str");
					Integer count = users.get(sid);
					if (count == null) {
						count = 1;
					}
					else {
						++count;
					}
					users.put(sid, count);
					count = usersOriginators.get(sid);
					if (count == null) {
						count = 1;
					}
					else {
						++count;
					}
					usersOriginators.put(sid, count);
				}
				JSONObject entities = (JSONObject) jv.get("entities");
				JSONObject retweeted_status = (JSONObject) jv
						.get("retweeted_status");
				if (retweeted_status != null) {
					JSONObject originalUser = (JSONObject)(retweeted_status.get("user"));
					String osid = (String)originalUser.get("id_str");
					Integer count = users.get(osid);
					if (count == null) {
						count = 1;
					}
					else {
						++count;
					}
					users.put(osid, count);
					if (currentUser != null) {
						
						UserPair up = new UserPair(osid, sid);
						count = userMap.get(up);
						if (count == null) {
							count = 1;
							userMap.put(up, count);
						}
						else {
							++count;
							userMap.put(up, count);
						}
					}
/*					System.out.println(id + " "
							+ retweeted_status.toJSONString());*/
					if (entities != null) {

						{

							JSONArray hashtags = (JSONArray) entities
									.get("hashtags");
							if (hashtags != null) {
								for (JSONObject h : hashtags
										.toArray(new JSONObject[hashtags.size()])) {
									String ht = h.get("text").toString()
											.toLowerCase();
									count = hashtagCount.get(ht);
									if (count == null) {
										count = 1;
									} else {
										count = count + 1;
									}
									hashtagCount.put(ht, count);
								}
							}
						}
						{

							/*if (retweeted_status != null) {
								System.out.println("Found Hashtags"
										+ hashtags.size());
								for (JSONObject h : hashtags
										.toArray(new JSONObject[hashtags.size()])) {
									String ht = h.get("text").toString();
									hashtagsFile.println(ht);
								}
							}*/
						}
					}
				}
			}
			for (Entry<String, Integer> se : hashtagCount.entrySet()) {
				hashtagsFile.printf("%d %s\n",se.getValue(), se.getKey());
			}
			for (Entry<UserPair, Integer> se : userMap.entrySet()) {
				usersMapFile.printf("%s %s %d\n",se.getKey().originalUser, se.getKey().user, se.getValue());
			}
			for (Entry<String, Integer> se : users.entrySet()) {
				usersFile.printf("%s %d\n",se.getKey(), se.getValue());
			}
			for (Entry<String, Integer> se : usersOriginators.entrySet()) {
				usersOriginatorsFile.printf("%s %d\n",se.getKey(), se.getValue());
			}
			hashtagsFile.close();
			usersFile.close();
			usersMapFile.close();
			usersOriginatorsFile.close();
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (ParseException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (UnsupportedEncodingException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}

	}

}
